{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 多分类问题中的混淆矩阵\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from matplotlib import pyplot as plt\n",
    "from sklearn import datasets\n",
    "\n",
    "digits = datasets.load_digits()\n",
    "X = digits.data\n",
    "y = digits.target                                       #10分类问题\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.8, random_state=666)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\annaconda\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.9408901251738526"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "log_reg = LogisticRegression()\n",
    "log_reg.fit(X_train, y_train)\n",
    "log_reg.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_predict = log_reg.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9408901251738526"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import precision_score\n",
    "#precision_score(y_test, y_predict)\n",
    "precision_score(y_test, y_predict, average=\"micro\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[148,   0,   1,   0,   0,   0,   0,   0,   0,   0],\n",
       "       [  0, 125,   2,   0,   0,   0,   0,   3,   2,  11],\n",
       "       [  0,   1, 134,   0,   0,   0,   0,   0,   1,   0],\n",
       "       [  0,   0,   1, 138,   0,   5,   0,   1,   4,   0],\n",
       "       [  2,   4,   0,   0, 138,   0,   1,   3,   0,   2],\n",
       "       [  1,   2,   1,   0,   0, 146,   1,   0,   0,   1],\n",
       "       [  0,   2,   0,   0,   0,   1, 132,   0,   1,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0, 135,   0,   1],\n",
       "       [  0,   8,   2,   1,   3,   3,   0,   1, 120,   2],\n",
       "       [  0,   1,   0,   6,   0,   1,   0,   1,   1, 137]], dtype=int64)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "confusion_matrix(y_test, y_predict)          #10分类的混淆矩阵"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x11dc1af748>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPoAAAECCAYAAADXWsr9AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAK0UlEQVR4nO3dT4hd5RnH8d/PmZE4MRJDQohJ6CgUWyOUyFCiAReOi7ZK3XRhIYG6mY3VKIJoNy67EdFFEUKsmwa7iFmUUNSCuugmdDIJaJwUJLaTmBhvE6JiCPPv6eJeaZJJc8+Y8865N8/3A0JmvHl9vN6v59zJOe91RAjAje2mpgcAUB6hAwkQOpAAoQMJEDqQAKEDCTQWuu2f2f6n7U9tv9DUHFXZ3mz7A9tTto/a3tX0TFXYHrB92PaBpmepwvZq2/tsH+s81/c3PVM3tp/tvCY+tv2W7RVNz3SlRkK3PSDpD5J+LukeSb+2fU8TsyzBnKTnIuLHkrZJerIPZpakXZKmmh5iCV6T9E5E/EjST9Tjs9veKOlpSaMRca+kAUmPNzvVYk0d0X8q6dOIOB4RM5L+LOmxhmapJCJOR8Rk59ffqP0C3NjsVNdme5OkRyTtaXqWKmzfJulBSW9IUkTMRMT5ZqeqZFDSLbYHJQ1LOtXwPIs0FfpGSScu+fqkejyaS9kekbRV0sFmJ+nqVUnPS1poepCK7pLUkvRm5+3GHtsrmx7qWiLic0kvS5qWdFrSVxHxXrNTLdZU6L7K9/riWlzbt0p6W9IzEfF10/P8P7YflfRlRBxqepYlGJR0n6TXI2KrpG8l9fTPb2zfrvbZ6J2S7pC00vaOZqdarKnQT0rafMnXm9SDpztXsj2kduR7I2J/0/N0sV3SL23/S+23Rg/Z/lOzI3V1UtLJiPjuTGmf2uH3soclfRYRrYiYlbRf0gMNz7RIU6H/Q9IPbd9p+2a1f3jxl4ZmqcS21X7vOBURrzQ9TzcR8WJEbIqIEbWf3/cjoueONJeKiC8knbB9d+dbY5I+aXCkKqYlbbM93HmNjKkHf4A42MQ/NCLmbP9W0rtq/5TyjxFxtIlZlmC7pJ2SPrJ9pPO930XEXxuc6Ub0lKS9nQPAcUlPNDzPNUXEQdv7JE2q/SczhyXtbnaqxcxtqsCNjyvjgAQIHUiA0IEECB1IgNCBBBoP3fZ40zMsRb/NKzHzcuj1eRsPXVJPP0FX0W/zSsy8HHp63l4IHUBhRS6YWbt2bYyMjFR6bKvV0rp16yo99tChfro/A2hGRCy6aazIJbAjIyOamJiofd32pcRllFqbKw/RCzh1BxIgdCABQgcSIHQgAUIHEqgUer/twQ7gcl1D79M92AFcosoRve/2YAdwuSqh9/Ue7ACqhV5pD3bb47YnbE+0Wq3rnwxAbaqEXmkP9ojYHRGjETFa9dp1AMujSuh9twc7gMt1vamlT/dgB3CJSnevdT6kgA8qAPoUV8YBCRA6kAChAwkQOpAAoQMJFNkc0naRjdKOHTtWYllJ0pYtW4qsOz8/X2RdXG5oaKjIugsLC0XWlaTVq1fXvub58+c1Nze36GpWjuhAAoQOJEDoQAKEDiRA6EAChA4kQOhAAoQOJEDoQAKEDiRA6EAChA4kQOhAAoQOJEDoQAKEDiRA6EAChA4kQOhAAoQOJEDoQAKEDiTQV9s924t2sa3NqVOLPvK9Fhs2bCiyLpZHyddcifY667LdM5ARoQMJEDqQAKEDCRA6kAChAwkQOpBA19Btb7b9ge0p20dt71qOwQDUZ7DCY+YkPRcRk7ZXSTpk+28R8Unh2QDUpOsRPSJOR8Rk59ffSJqStLH0YADqs6T36LZHJG2VdLDEMADKqHLqLkmyfauktyU9ExFfX+Xvj0sar3E2ADWpdFOL7SFJByS9GxGvVHg8N7V0cFNLf0tzU4vb/6ZvSJqqEjmA3lPlPfp2STslPWT7SOevXxSeC0CNur5Hj4i/Syp3/gKgOK6MAxIgdCABQgcSIHQgAUIHEuirXWBLKnVhxNmzZ4usK0lr1qwpsu6KFSuKrHvx4sUi60rl/vsNDQ0VWVeSZmZmiqzLLrBAUoQOJEDoQAKEDiRA6EAChA4kQOhAAoQOJEDoQAKEDiRA6EAChA4kQOhAAoQOJEDoQAKEDiRA6EAChA4kQOhAAoQOJEDoQAKEDiRQbLvngYGB2tctseZ3Sm29W9K5c+eKrFtqG+mSnzU+ONj180K/l9nZ2SLrSmVez/Pz82z3DGRF6EAChA4kQOhAAoQOJEDoQAKEDiRQOXTbA7YP2z5QciAA9VvKEX2XpKlSgwAop1LotjdJekTSnrLjACih6hH9VUnPS1ooOAuAQrqGbvtRSV9GxKEujxu3PWF7orbpANSi600ttn8vaaekOUkrJN0maX9E7LjG7+GmlmXATS3/w00tbd/7ppaIeDEiNkXEiKTHJb1/rcgB9B7+HB1IYEnnOxHxoaQPi0wCoBiO6EAChA4kQOhAAoQOJEDoQALFdoEtcXHETTeV+//SwkKZq3tLPL+lzc3NFVl3aGioyLpSfz7PJRqJCHaBBbIidCABQgcSIHQgAUIHEiB0IAFCBxIgdCABQgcSIHQgAUIHEiB0IAFCBxIgdCABQgcSIHQgAUIHEiB0IAFCBxIgdCABQgcSKPNZs+q/XTn7bd6SSu3WOj09XWRdSdq8eXORdUt+1PNyvuY4ogMJEDqQAKEDCRA6kAChAwkQOpAAoQMJVArd9mrb+2wfsz1l+/7SgwGoT9ULZl6T9E5E/Mr2zZKGC84EoGZdQ7d9m6QHJf1GkiJiRtJM2bEA1KnKqftdklqS3rR92PYe2ysLzwWgRlVCH5R0n6TXI2KrpG8lvXDlg2yP256wPVHzjACuU5XQT0o6GREHO1/vUzv8y0TE7ogYjYjROgcEcP26hh4RX0g6YfvuzrfGJH1SdCoAtar6U/enJO3t/MT9uKQnyo0EoG6VQo+II5I4JQf6FFfGAQkQOpAAoQMJEDqQAKEDCRA6kIBLbDlrm72TsazOnDlTZN3169cXWVcqs5V0RCgiFi3MER1IgNCBBAgdSIDQgQQIHUiA0IEECB1IgNCBBAgdSIDQgQQIHUiA0IEECB1IgNCBBAgdSIDQgQQIHUiA0IEECB1IgNCBBAgdSKCvdoFdtWpViWUlSRcuXCiy7sLCQpF1JWlwsOqH4faG2dnZYmuX2FFVkiYnJ4usK0mjo/V/bun8/Dy7wAJZETqQAKEDCRA6kAChAwkQOpAAoQMJVArd9rO2j9r+2PZbtleUHgxAfbqGbnujpKcljUbEvZIGJD1eejAA9al66j4o6Rbbg5KGJZ0qNxKAunUNPSI+l/SypGlJpyV9FRHvlR4MQH2qnLrfLukxSXdKukPSSts7rvK4cdsTtifqHxPA9ahy6v6wpM8iohURs5L2S3rgygdFxO6IGI2I+q/UB3BdqoQ+LWmb7WG3bxEakzRVdiwAdaryHv2gpH2SJiV91Pk9uwvPBaBGlW5ojoiXJL1UeBYAhXBlHJAAoQMJEDqQAKEDCRA6kAChAwn01XbPpbb0laQSz4MkDQ8PF1lXKrdFdannudRzLJV9bZTSarVqX3NsbExHjhxhu2cgI0IHEiB0IAFCBxIgdCABQgcSIHQgAUIHEiB0IAFCBxIgdCABQgcSIHQgAUIHEiB0IAFCBxIgdCABQgcSIHQgAUIHEiB0IIFSu8C2JP274sPXSvpP7UOU02/zSsy8HHpl3h9ExLorv1kk9KWwPRERo40OsQT9Nq/EzMuh1+fl1B1IgNCBBHoh9N1ND7BE/TavxMzLoafnbfw9OoDyeuGIDqAwQgcSIHQgAUIHEiB0IIH/AucJrxLYoQYiAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 288x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "cfm = confusion_matrix(y_test, y_predict)\n",
    "plt.matshow(cfm, cmap=plt.cm.gray)         #映射成灰度图  cmap = color map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x11dc2efa88>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPoAAAECCAYAAADXWsr9AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAK5UlEQVR4nO3dQYhd5RnG8edxZsQkVlTSTMZEGsViK5KSOBQ14MK4SBvRTRcWFOomm1YTCRjtxrUgEhdFGGKDYNBFzKJosRaNi25Cx4ygcSyK2jjxTiYN1IguMkneLuYKyczUe64535x78/5/IGSO18+Xyfxzzr0597uOCAG4tF3W9AAAyiN0IAFCBxIgdCABQgcSIHQggcZCt73F9r9sf2L7iabmqMr29bYP2p60fcT29qZnqsL2gO0J2681PUsVtq+2vd/2R+3v9R1Nz9SJ7cfaPxMf2H7Z9hVNzzRfI6HbHpD0J0m/knSLpN/avqWJWbpwRtLOiPi5pNsl/b4PZpak7ZImmx6iC89JeiMifibpF+rx2W2vkfSopNGIuFXSgKQHmp1qoabO6L+U9ElEfBoRpyW9Iun+hmapJCJaEXG4/euvNfcDuKbZqb6f7bWStkra0/QsVdi+StJdkl6QpIg4HRH/bXaqSgYlLbM9KGm5pC8bnmeBpkJfI+mL876eUo9Hcz7b6yRtkHSo2Uk62i3pcUnnmh6kohslnZC0t/10Y4/tFU0P9X0i4pikZyQdldSS9FVEvNnsVAs1FboXOdYX9+LavlLSq5J2RMSppuf5f2zfK2kmIt5tepYuDEraKOn5iNgg6RtJPf36je1rNHc1eoOk6yStsP1gs1Mt1FToU5KuP+/rterBy535bA9pLvJ9EXGg6Xk62CTpPtufa+6p0d22X2p2pI6mJE1FxHdXSvs1F34vu0fSZxFxIiJmJR2QdGfDMy3QVOj/lPRT2zfYvlxzL178paFZKrFtzT13nIyIZ5uep5OIeDIi1kbEOs19f9+OiJ4705wvIqYlfWH75vahzZI+bHCkKo5Kut328vbPyGb14AuIg038TyPijO0/SPqb5l6l/HNEHGlili5skvSQpPdtv9c+9seI+GuDM12KHpG0r30C+FTSww3P870i4pDt/ZIOa+5vZiYkjTU71ULmbarApY8744AECB1IgNCBBAgdSIDQgQQaD932tqZn6Ea/zSsx81Lo9XkbD11ST3+DFtFv80rMvBR6et5eCB1AYUVumLHdd3fhDA0NVXrcuXPndNll1f98nJ2d/aEjAT9IRCx401gjt8D2opUrVxZZt9VqFVkX6AaX7kAChA4kQOhAAoQOJEDoQAKVQu+3PdgBXKhj6H26BzuA81Q5o/fdHuwALlQl9L7egx1AtTvjKu3B3n73Tk/f2A9kVSX0SnuwR8SY2rtf9uO97sClrMqle9/twQ7gQh3P6H26BzuA81R691r7Qwr4oAKgT3FnHJAAoQMJEDqQAKEDCRA6kAB7xrWNjIwUWZc945bG1q1bi6x77NixIutK0sTERO1rjo6OLnqcMzqQAKEDCRA6kAChAwkQOpAAoQMJEDqQAKEDCRA6kAChAwkQOpAAoQMJEDqQAKEDCRA6kAChAwkQOpAAoQMJEDqQAKEDCRA6kAChAwk4ov6PMi/1+ejDw8MllpUkHT9+vNja6F+rV68utvb09HSRdSPC849xRgcSIHQgAUIHEiB0IAFCBxIgdCABQgcS6Bi67ettH7Q9afuI7e1LMRiA+lT5fPQzknZGxGHbP5L0ru2/R8SHhWcDUJOOZ/SIaEXE4favv5Y0KWlN6cEA1Ker5+i210naIOlQiWEAlFHl0l2SZPtKSa9K2hERpxb599skbatxNgA1qRS67SHNRb4vIg4s9piIGJM01n58kTe1APhhqrzqbkkvSJqMiGfLjwSgblWeo2+S9JCku22/1/7n14XnAlCjjpfuEfEPSQve3wqgf3BnHJAAoQMJEDqQAKEDCRA6kEBf7QJb0sjISJF1W61WkXVL2rFjR5F1d+/eXWRdqdzv35YtW4qsK0l79+4tsi67wAJJETqQAKEDCRA6kAChAwkQOpAAoQMJEDqQAKEDCRA6kAChAwkQOpAAoQMJEDqQAKEDCRA6kAChAwkQOpAAoQMJEDqQAKEDCRA6kECR7Z6XLVsWN910U+3r3nbbbbWv+Z0XX3yx2NqYU2pLZknauHFjkXVff/31IutK0vr162tf8+OPP9a3337Lds9ARoQOJEDoQAKEDiRA6EAChA4kQOhAApVDtz1ge8L2ayUHAlC/bs7o2yVNlhoEQDmVQre9VtJWSXvKjgOghKpn9N2SHpd0ruAsAArpGLrteyXNRMS7HR63zfa47fGzZ8/WNiCAi1fljL5J0n22P5f0iqS7bb80/0ERMRYRoxExOjAwUPOYAC5Gx9Aj4smIWBsR6yQ9IOntiHiw+GQAasPfowMJDHbz4Ih4R9I7RSYBUAxndCABQgcSIHQgAUIHEiB0IIEiu8AODQ3FtddeW/u6JXcRnZmZKbJuq9Uqsm4/Kvn714/f5+Hh4drXPHnypGZnZ9kFFsiI0IEECB1IgNCBBAgdSIDQgQQIHUiA0IEECB1IgNCBBAgdSIDQgQQIHUiA0IEECB1IgNCBBAgdSIDQgQQIHUiA0IEECB1IoKvPXqvqzJkzRXZVZRfRpbFq1aoi6/bj93j16tXF1p6eni629nyc0YEECB1IgNCBBAgdSIDQgQQIHUiA0IEEKoVu+2rb+21/ZHvS9h2lBwNQn6o3zDwn6Y2I+I3tyyUtLzgTgJp1DN32VZLukvQ7SYqI05JOlx0LQJ2qXLrfKOmEpL22J2zvsb2i8FwAalQl9EFJGyU9HxEbJH0j6Yn5D7K9zfa47fGaZwRwkaqEPiVpKiIOtb/er7nwLxARYxExGhGjdQ4I4OJ1DD0ipiV9Yfvm9qHNkj4sOhWAWlV91f0RSfvar7h/KunhciMBqFul0CPiPUlckgN9ijvjgAQIHUiA0IEECB1IgNCBBAgdSMARUf+idv2LApeY4eHh2tc8efKkZmdnPf84Z3QgAUIHEiB0IAFCBxIgdCABQgcSIHQgAUIHEiB0IAFCBxIgdCABQgcSIHQgAUIHEiB0IAFCBxIgdCABQgcSIHQgAUIHEiB0IIGqn6baEw4ePFhs7Z07dxZZd2pqqsi6krR+/foi65bYGViS3nrrrSLrStLIyEiRdVutVpF1pTK7wJ46dWrR45zRgQQIHUiA0IEECB1IgNCBBAgdSIDQgQQqhW77MdtHbH9g+2XbV5QeDEB9OoZue42kRyWNRsStkgYkPVB6MAD1qXrpPihpme1BScslfVluJAB16xh6RByT9Iyko5Jakr6KiDdLDwagPlUu3a+RdL+kGyRdJ2mF7QcXedw22+O2x+sfE8DFqHLpfo+kzyLiRETMSjog6c75D4qIsYgYjYjRuocEcHGqhH5U0u22l9u2pM2SJsuOBaBOVZ6jH5K0X9JhSe+3/5uxwnMBqFGl96NHxFOSnio8C4BCuDMOSIDQgQQIHUiA0IEECB1IgNCBBFxia1/bRfYLLrE97neOHz9eZN2nn366yLqStGvXriLrrlq1qsi6MzMzRdaVym33XGrra0manp4usm5EeP4xzuhAAoQOJEDoQAKEDiRA6EAChA4kQOhAAoQOJEDoQAKEDiRA6EAChA4kQOhAAoQOJEDoQAKEDiRA6EAChA4kQOhAAoQOJEDoQAKldoE9IenfFR++UtJ/ah+inH6bV2LmpdAr8/4kIn48/2CR0LthezwiRhsdogv9Nq/EzEuh1+fl0h1IgNCBBHoh9LGmB+hSv80rMfNS6Ol5G3+ODqC8XjijAyiM0IEECB1IgNCBBAgdSOB/A4uB3UOJG0gAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 288x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "row_sums = np.sum(cfm, axis=1)         #每行的样本和\n",
    "err_matrix = cfm/row_sums              #百分比\n",
    "np.fill_diagonal(err_matrix, 0)        #对角线置零，每个格子犯错的百分比\n",
    "plt.matshow(err_matrix, cmap=plt.cm.gray) #越亮的地方是犯错越多的地方\n",
    "\n",
    "# 第二行最后一个：有很多真值为1，但被预测成了9；可以调整预测错误多的数据的阈值来获得更好的结果"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
